import json
import re
import time
import random
import os  # 新增os模块获取用户路径
import pandas as pd
from selenium import webdriver
from selenium.webdriver import ActionChains
from selenium.webdriver.common.by import By
from selenium.webdriver.edge.service import Service
from selenium.webdriver.edge.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException, WebDriverException, NoSuchElementException
from webdriver_manager.microsoft import EdgeChromiumDriverManager
import numpy as np
import psutil
import base64


# ======== 核心修改1：复用用户浏览器配置文件 ========
def get_edge_user_data_dir():
    """获取Edge用户数据目录路径"""
    username = os.getlogin()
    return f"C:\\Users\\{username}\\AppData\\Local\\Microsoft\\Edge\\User Data"

def close_existing_edge_browsers():
    """关闭所有正在运行的Edge浏览器进程"""
    print("检查正在运行的Edge浏览器进程...")
    edge_closed = False
    for proc in psutil.process_iter(['name']):
        try:
            # 检查进程名是否为msedge.exe（不区分大小写）
            if proc.info['name'].lower() == "msedge.exe":
                print(f"关闭Edge进程 (PID: {proc.pid})")
                proc.kill()
                edge_closed = True
        except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess):
            continue

    if edge_closed:
        print("所有Edge浏览器进程已关闭")
        time.sleep(2)  # 等待进程完全终止
    else:
        print("没有检测到正在运行的Edge浏览器")

# ======== 人类行为模拟模块 ========
def human_typing(element, text):
    """模拟人类打字行为（带随机错误和修正）"""
    for char in text:
        # 随机决定是否打错字（5%概率）
        if random.random() < 0.05:
            wrong_char = random.choice('abcdefghijklmnopqrstuvwxyz')
            element.send_keys(wrong_char)
            time.sleep(random.uniform(0.1, 0.3))
            element.send_keys('\b')  # 退格删除

        element.send_keys(char)

        # 随机输入间隔
        pause_type = random.choices(
            ['short', 'medium', 'long'],
            weights=[0.7, 0.25, 0.05]
        )[0]

        if pause_type == 'short':
            time.sleep(random.uniform(0.05, 0.15))
        elif pause_type == 'medium':
            time.sleep(random.uniform(0.2, 0.5))
        else:  # long pause
            time.sleep(random.uniform(1.0, 2.0))


def mouse_movement_simulation(driver, target_element=None):
    """模拟人类鼠标移动轨迹（贝塞尔曲线）"""
    action = ActionChains(driver)
    width = driver.execute_script("return window.innerWidth")
    height = driver.execute_script("return window.innerHeight")

    # 随机起点
    start_x = random.randint(0, width // 3)
    start_y = random.randint(height // 4, height * 3 // 4)
    action.move_by_offset(start_x, start_y).perform()

    # 创建控制点
    if target_element:
        target_loc = target_element.location
        target_x = target_loc['x'] + random.randint(0, target_element.size['width'] // 2)
        target_y = target_loc['y'] + random.randint(0, target_element.size['height'] // 2)
        control_x = random.randint(start_x, target_x)
        control_y = random.randint(start_y, target_y)
    else:
        target_x = width // 2
        target_y = height // 2
        control_x = random.randint(width // 4, width * 3 // 4)
        control_y = random.randint(height // 4, height * 3 // 4)

    # 贝塞尔曲线路径
    steps = 30
    for t in np.linspace(0, 1, steps):
        # 二次贝塞尔曲线公式
        x = (1 - t) ** 2 * start_x + 2 * (1 - t) * t * control_x + t ** 2 * target_x
        y = (1 - t) ** 2 * start_y + 2 * (1 - t) * t * control_y + t ** 2 * target_y

        # 随机速度变化
        if t < 0.8:
            time.sleep(random.uniform(0.01, 0.05))
        else:
            time.sleep(random.uniform(0.05, 0.15))

        # 计算相对移动距离
        current_x, current_y = action.driver.execute_script(
            "return [window.scrollX + window.innerWidth/2, window.scrollY + window.innerHeight/2]")
        move_x = x - current_x
        move_y = y - current_y

        # 小步移动
        action.move_by_offset(move_x, move_y).perform()

    # 最终微调
    if target_element:
        for _ in range(random.randint(1, 3)):
            dx = random.randint(-5, 5)
            dy = random.randint(-5, 5)
            action.move_by_offset(dx, dy).pause(random.uniform(0.1, 0.3)).perform()


def human_click(driver, element):
    """增强版人类点击模拟"""
    # 先模拟鼠标移动
    mouse_movement_simulation(driver, element)

    # 悬停观察
    ActionChains(driver).move_to_element(element).pause(random.uniform(0.5, 1.5)).perform()

    # 点击前可能的微小移动
    for _ in range(random.randint(0, 2)):
        offset_x = random.randint(-3, 3)
        offset_y = random.randint(-3, 3)
        ActionChains(driver).move_by_offset(offset_x, offset_y).pause(0.1).perform()

    # 真实点击
    element.click()

    # 点击后停留时长
    time.sleep(random.uniform(0.8, 2.5))


def random_scroll(driver):
    """模拟人类滚动行为"""
    scroll_distance = random.randint(200, 800)
    scroll_steps = random.randint(3, 8)
    scroll_direction = random.choice([-1, 1])  # 向上或向下

    for _ in range(scroll_steps):
        driver.execute_script(f"window.scrollBy(0, {scroll_distance * scroll_direction});")
        time.sleep(random.uniform(0.3, 1.2))

        # 30%概率改变方向
        if random.random() < 0.3:
            scroll_direction *= -1


# ======== 结束人类行为模拟模块 ========

def execute_cdp_command(driver, command, params={}):
    """执行Chrome DevTools Protocol命令"""
    resource = f"/session/{driver.session_id}/chromium/send_command_and_get_result"
    url = driver.command_executor._url + resource
    body = json.dumps({'cmd': command, 'params': params})
    response = driver.command_executor._request('POST', url, body)
    return response.get('value')


def apply_stealth_mode(driver):
    """应用深度反检测策略对抗WoS反爬系统"""
    # 1. WebDriver属性伪装
    driver.execute_cdp_cmd('Page.addScriptToEvaluateOnNewDocument', {
        'source': '''
            // 核心属性覆盖（强化版）
            Object.defineProperty(navigator, 'webdriver', {
                get: () => undefined,
                configurable: true,
                enumerable: true
            });

            // 移除自动化痕迹
            window.chrome = {};
            window.chrome.runtime = {};

            // 修正插件列表
            const originalPlugins = Array.from(navigator.plugins);
            Object.defineProperty(navigator, 'plugins', {
                get: () => originalPlugins,
                configurable: false
            });

            // 修正语言设置
            Object.defineProperty(navigator, 'languages', {
                get: () => ['zh-CN', 'zh', 'en-US', 'en'],
                configurable: false
            });

            // 修复时区
            Object.defineProperty(Date.prototype, 'getTimezoneOffset', {
                value: function() {
                    return new Date().getTimezoneOffset();
                },
                configurable: false
            });

            // 保护原生函数
            const originalFunction = Function.prototype.toString;
            Function.prototype.toString = function() {
                if (this.name.includes('webdriver')) {
                    return 'function() { [native code] }';
                }
                return originalFunction.call(this);
            };

            // 覆盖webgl参数
            const getParameter = WebGLRenderingContext.prototype.getParameter;
            WebGLRenderingContext.prototype.getParameter = function(parameter) {
                // 屏蔽webdriver参数
                if (parameter === 37445) {
                    return 'Intel Inc.'; // 伪造的GPU供应商
                }
                if (parameter === 37446) {
                    return 'Intel Iris OpenGL Engine'; // 伪造的GPU渲染器
                }
                return getParameter(parameter);
            };
        '''
    })

    # 2. 隐藏WebDriver特征
    driver.execute_cdp_cmd('Network.setUserAgentOverride', {
        'userAgent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36 Edg/125.0.0.0',
        'platform': 'Win32'
    })

    # 3. 覆盖navigator属性
    driver.execute_cdp_cmd('Page.addScriptToEvaluateOnNewDocument', {
        'source': '''
            Object.defineProperty(navigator, 'maxTouchPoints', {get: () => 5});
            Object.defineProperty(navigator, 'hardwareConcurrency', {get: () => 8});
            Object.defineProperty(navigator, 'deviceMemory', {get: () => 8});
        '''
    })

    # 4. 禁用自动化控制的Blink特性
    driver.execute_cdp_cmd('Page.addScriptToEvaluateOnNewDocument', {
        'source': '''
            Object.defineProperty(window, 'callPhantom', {value: undefined, enumerable: false});
            Object.defineProperty(window, '_phantom', {value: undefined, enumerable: false});
            Object.defineProperty(window, 'phantom', {value: undefined, enumerable: false});
        '''
    })

    print("深度反检测策略已应用")

def setup_browser_options():
    """配置优化的浏览器选项"""
    close_existing_edge_browsers()

    print("配置Edge浏览器（深度反检测模式）...")
    edge_options = Options()

    # 复用现有浏览器配置文件
    user_data_dir = get_edge_user_data_dir()
    edge_options.add_argument(f"user-data-dir={user_data_dir}")
    edge_options.add_argument("profile-directory=Default")

    # 关键反检测参数（针对WoS优化）
    edge_options.add_argument("--disable-blink-features=AutomationControlled")
    edge_options.add_argument("--disable-infobars")
    edge_options.add_argument("--disable-notifications")
    edge_options.add_argument("--disable-popup-blocking")
    edge_options.add_argument("--disable-dev-shm-usage")
    edge_options.add_argument("--no-sandbox")
    edge_options.add_argument("--disable-setuid-sandbox")
    edge_options.add_argument("--disable-web-security")
    edge_options.add_argument("--allow-running-insecure-content")
    edge_options.add_argument("--ignore-certificate-errors")
    edge_options.add_argument("--disable-gpu")
    edge_options.add_argument("--disable-3d-apis")

    # 禁用自动化提示
    edge_options.add_experimental_option("excludeSwitches", ["enable-automation", "enable-logging"])
    edge_options.add_experimental_option('useAutomationExtension', False)

    # 禁用自动化控制标志
    edge_options.set_capability("goog:loggingPrefs", {'performance': 'ALL'})
    edge_options.set_capability("goog:chromeOptions", {'w3c': False, 'args': edge_options.arguments})

    # 使用随机端口避免检测
    edge_options.add_argument(f"--remote-debugging-port={random.randint(9000, 9999)}")

    # 添加随机指纹参数
    edge_options.add_argument(
        f"--user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36 Edg/125.0.0.0")
    edge_options.add_argument(f"--window-size={random.randint(1200, 1400)},{random.randint(800, 1000)}")

    return edge_options


def validate_wos_page(driver):
    """验证是否成功加载WoS页面"""
    try:
        # 检查是否有反爬检测页面
        if "trueHostName" in driver.page_source or "snowplow.js" in driver.page_source:
            print("检测到反爬页面，尝试绕过...")
            return False

        # 检查核心元素是否存在
        WebDriverWait(driver, 15).until(
            EC.presence_of_element_located((By.XPATH, "//input[contains(@title, 'Search')]"))
        )
        print("核心搜索框加载成功")
        return True
    except TimeoutException:
        print("页面元素验证失败")
        return False


def scrape_wos_no_proxy(author_address, max_retries=3):
    """增强反爬能力的WOS爬取函数"""
    retries = 0
    driver = None

    while retries <= max_retries:
        try:
            edge_options = setup_browser_options()

            # 安装并启动Edge驱动
            print("使用Edge驱动管理器...")
            service = Service(EdgeChromiumDriverManager().install())

            # 关键：隐藏驱动特征
            service.creation_flags = 0x8000000  # CREATE_NO_WINDOW

            print("启动浏览器...")
            driver = webdriver.Edge(service=service, options=edge_options)

            # 不最大化窗口以避免检测
            driver.set_window_size(random.randint(1200, 1400), random.randint(800, 1000))
            print(f"浏览器窗口大小: {driver.get_window_size()}")

            # 应用深度反检测策略
            apply_stealth_mode(driver)

            # ========= 访问Web of Science =========
            print("正在加载Web of Science...")

            # 随机选择访问方法
            access_methods = [
                "https://www.webofscience.com/wos/woscc/basic-search",
                "https://www.webofscience.com/wos/woscc/advanced-search",
                "https://www.webofscience.com/"
            ]

            initial_url = random.choice(access_methods)
            print(f"使用访问方法: {initial_url}")
            driver.get(initial_url)

            # 添加随机延迟模拟人类思考
            time.sleep(random.uniform(3, 7))

            # 随机滚动页面
            scroll_distance = random.randint(200, 800)
            driver.execute_script(f"window.scrollBy(0, {scroll_distance});")
            time.sleep(random.uniform(1, 2))

            # 验证页面是否成功加载
            if not validate_wos_page(driver):
                raise TimeoutException("反爬检测页面未绕过")

            # 保存页面截图验证
            timestamp = int(time.time())
            screenshot_name = f"wos_success_{timestamp}.png"
            driver.save_screenshot(screenshot_name)
            print(f"成功页面截图已保存: {screenshot_name}")

            # 处理Cookie同意弹窗
            try:
                    cookie_btn = WebDriverWait(driver, 15).until(
                        EC.element_to_be_clickable((By.XPATH, '//*[@id="onetrust-close-btn-container"]/button'))
                    )
                    human_click(driver, cookie_btn)
                    print("已关闭Cookie弹窗")
                    time.sleep(random.uniform(1.5, 3.0))
            except:
                    print("未检测到Cookie弹窗")

                # 确保页面加载完成
            WebDriverWait(driver, 20).until(
                    EC.visibility_of_element_located((By.XPATH, '//*[@id="snSearchType"]'))
                )

                # 随机滚动增加真实感
            random_scroll(driver)

                # ===================== 搜索操作 =====================
            print("开始模拟人类搜索流程...")
                # 选择搜索类型
            search_type_btn = WebDriverWait(driver, 25).until(
                    EC.element_to_be_clickable((By.XPATH,
                                                '//*[@id="snSearchType"]/div[1]/app-search-row/div/div[1]/app-select-search-field/wos-select/button'))
                )
            human_click(driver, search_type_btn)
            print("已打开搜索类型菜单")
            time.sleep(random.uniform(2.0, 4.0))

                # 随机滚动增加真实感
            random_scroll(driver)

                # 切换到"地址"搜索
            address_option = WebDriverWait(driver, 15).until(
                    EC.element_to_be_clickable(
                        (By.XPATH, '//*[@id="global-select"]/div[1]/div/div[9]'))
                )
            human_click(driver, address_option)
            print("已选择'地址'搜索")
            time.sleep(random.uniform(3, 5))

                # 输入作者地址（人类打字模拟）
            search_input = WebDriverWait(driver, 15).until(
                    EC.visibility_of_element_located((By.ID, "search-option"))
                )
            print("模拟人类输入地址信息...")
            human_typing(search_input, author_address)
            time.sleep(random.uniform(1.5, 3.0))

                # 提交搜索
            search_btn = WebDriverWait(driver, 15).until(
                    EC.element_to_be_clickable((By.XPATH,
                                                '//*[@id="snSearchType"]/div[3]/button[2]/span[1]'))
                )
            human_click(driver, search_btn)
            print("搜索请求已提交...")

                # 人类等待行为
            search_delay = random.choices(
                    [random.uniform(5, 8), random.uniform(8, 15), random.uniform(15, 25)],
                    weights=[0.6, 0.3, 0.1]
                )[0]
            print(f"模拟人类等待行为：{search_delay:.1f} 秒...")
            time.sleep(search_delay)

                # 验证处理
            verification_attempts = 0
            while "verification" in driver.current_url.lower() or "captcha" in driver.page_source.lower():
                    if verification_attempts < 2:
                        wait_time = 10 * (2 ** verification_attempts)
                        print(f"检测到验证页面（尝试 {verification_attempts + 1}/3），等待 {wait_time} 秒...")
                        time.sleep(wait_time)
                        driver.refresh()
                        verification_attempts += 1
                    else:
                        print("需要手动干预解决验证")
                        input("请手动完成验证后按回车继续...")
                        break
                    time.sleep(5)

                # 等待结果加载
            print("等待搜索结果...")
            random_scroll(driver)
            WebDriverWait(driver, 35).until(
                    EC.visibility_of_element_located((By.CSS_SELECTOR, "app-records-list"))
                )
            print("结果页面加载完成")
            time.sleep(random.uniform(3, 6))

                # 获取论文总数
            try:
                    result_count = driver.find_element(By.CSS_SELECTOR, "p.summary-title span").text
                    print(f"共找到 {result_count} 篇匹配论文")
            except:
                    print("无法获取结果总数")

            all_papers = []
            page = 1
            MAX_PAGES = 3

            while page <= MAX_PAGES:
                    print(f"处理第 {page} 页...")
                    time.sleep(random.uniform(4, 8))

                    # 提取当前页论文
                    papers = WebDriverWait(driver, 25).until(
                        EC.presence_of_all_elements_located((By.CSS_SELECTOR, "app-summary-record"))
                    )
                    print(f"本页发现 {len(papers)} 篇论文")

                    for idx, paper in enumerate(papers, 1):
                        try:
                            # 人类滚动行为
                            if idx % 3 == 0:
                                random_scroll(driver)

                            # 滚动到元素
                            driver.execute_script(
                                "arguments[0].scrollIntoView({behavior: 'smooth', block: 'center', inline: 'center'});",
                                paper
                            )
                            time.sleep(random.uniform(0.8, 1.5))

                            # 随机决定是否展开
                            if random.random() > 0.3:  # 70%概率展开
                                try:
                                    expand_btn = paper.find_element(By.CSS_SELECTOR, "button.title-link")
                                    if "展开" in expand_btn.text or "Expand" in expand_btn.text:
                                        human_click(driver, expand_btn)
                                        time.sleep(random.uniform(1.0, 2.5))
                                except:
                                    pass

                            # 提取标题
                            title = paper.find_element(By.CSS_SELECTOR, "a.title").text.strip()

                            # 提取作者
                            try:
                                authors = paper.find_element(By.CSS_SELECTOR, "div.authors").text.replace('作者',
                                                                                                          '').strip()
                            except:
                                authors = "N/A"

                            # 提取摘要
                            try:
                                abstract = paper.find_element(By.CSS_SELECTOR, "p.abstract-content").text.strip()
                            except:
                                abstract = "N/A"

                            # 提取联系信息
                            try:
                                author_info = paper.find_element(By.CSS_SELECTOR, "app-authors-addresses").text
                                emails = ';'.join(re.findall(r'[\w\.-]+@[\w\.-]+\.\w+', author_info))
                                addresses = author_info.split('查看所有组织')[0].strip().replace('\n', '; ')
                            except:
                                emails = "N/A"
                                addresses = "N/A"

                            all_papers.append({
                                "Title": title,
                                "Authors": authors,
                                "Abstract": abstract,
                                "Emails": emails,
                                "Addresses": addresses
                            })
                            print(f"  已提取第{idx}篇: {title[:45]}{'...' if len(title) > 45 else ''}")

                        except Exception as e:
                            print(f"  处理论文 {idx} 时出错：{str(e)}")
                            continue

                    # 翻页逻辑
                    try:
                        next_btn = driver.find_element(By.CSS_SELECTOR,
                                                       "button[title='下一页']:not([disabled]), button[title='Next page']:not([disabled])")

                        # 随机决定是否翻页
                        if random.random() < 0.85:  # 85%概率翻页
                            human_click(driver, next_btn)
                            print("已翻到下一页")

                            # 人类等待行为
                            page_wait = random.choices(
                                [random.uniform(5, 8), random.uniform(8, 12), random.uniform(12, 18)],
                                weights=[0.7, 0.25, 0.05]
                            )[0]
                            print(f"翻页后等待 {page_wait:.1f} 秒...")
                            time.sleep(page_wait)
                            page += 1

                            # 随机滚动
                            random_scroll(driver)

                            # 验证检查
                            if "verification" in driver.current_url.lower() or "captcha" in driver.page_source.lower():
                                print("检测到验证页面，启动自动处理...")
                                time.sleep(15)
                                if "verification" in driver.current_url.lower() or "captcha" in driver.page_source.lower():
                                    print("需要手动干预")
                                    input("请手动完成验证后按回车继续...")
                        else:
                            print("模拟人类跳过页面，终止爬取")
                            break

                    except Exception as e:
                        print(f"翻页失败：{str(e)}，终止爬取")
                        break

                # 保存数据
            df = pd.DataFrame(all_papers)
            if not df.empty:
                    output_file = f"WOS_{author_address.replace(' ', '_')}_论文数据.xlsx"
                    df.to_excel(output_file, index=False)
                    print(f"爬取完成！共保存 {len(all_papers)} 篇论文")
                    return True
            else:
                    print("未找到论文数据")
                    return False



        except Exception as e:

            print(f"发生异常: {str(e)}")

            # 保存错误截图

            if 'driver' in locals():

                try:

                    timestamp = int(time.time())

                    driver.save_screenshot(f"wos_error_{timestamp}.png")

                    print(f"错误截图已保存: wos_error_{timestamp}.png")

                    # 获取页面源代码用于调试

                    with open(f"wos_source_{timestamp}.html", "w", encoding="utf-8") as f:

                        f.write(driver.page_source)

                except:

                    print("无法保存截图或页面源代码")

            retries += 1

            if retries <= max_retries:

                print(f"第 {retries} 次重试...")

                time.sleep(10)  # 重试前等待

            else:

                print(f"已达到最大重试次数 ({max_retries})")

                raise


        finally:

            # 确保浏览器关闭

            if 'driver' in locals():

                try:

                    driver.quit()

                    print("浏览器已关闭")

                except:

                    print("关闭浏览器时出错")


if __name__ == "__main__":
    # 示例：爬取"北京大学"相关论文
    target_institution = "Peking University"
    print(f"开始爬取: {target_institution}")
    result = scrape_wos_no_proxy(target_institution)
    if result:
        print("任务成功")
    else:
        print("任务失败")